#include <stdlib.h>
#include <string.h>
#include <stdio.h>
#include <stdarg.h>
#include <assert.h>
#include <ctype.h>
#include <string>
#include <vector>
#include <map>
#include <algorithm>
#include "ILexer.h"
#include "Scintilla.h"
#include "SciLexer.h"
#include "WordList.h"
#include "LexAccessor.h"
#include "Accessor.h"
#include "StyleContext.h"
#include "CharacterSet.h"
#include "LexerModule.h"
#include "OptionSet.h"
#include "SparseState.h"
#include "SubStyles.h"

#ifdef SCI_NAMESPACE
using namespace Scintilla;
#endif

namespace {
  bool IsSpaceEquiv( int state ) {
    return ( state <= SCE_C_COMMENTDOC ) ||
           ( state == SCE_C_COMMENTLINEDOC ) || ( state == SCE_C_COMMENTDOCKEYWORD ) ||
           ( state == SCE_C_COMMENTDOCKEYWORDERROR );
  }

  bool FollowsPostfixOperator( StyleContext &sc, LexAccessor &styler ) {
    int pos = ( int ) sc.currentPos;
    while( --pos > 0 ) {
      char ch = styler[pos];
      if( ch == '+' || ch == '-' ) {
        return styler[pos - 1] == ch;
      }
    }
    return false;
  }

  bool followsReturnKeyword( StyleContext &sc, LexAccessor &styler ) {
    int pos = ( int ) sc.currentPos;
    int currentLine = styler.GetLine( pos );
    int lineStartPos = styler.LineStart( currentLine );
    while( --pos > lineStartPos ) {
      char ch = styler.SafeGetCharAt( pos );
      if( ch != ' ' && ch != '\t' ) {
        break;
      }
    }
    const char *retBack = "nruter";
    const char *s = retBack;
    while( *s
           && pos >= lineStartPos
           && styler.SafeGetCharAt( pos ) == *s ) {
      s++;
      pos--;
    }
    return !*s;
  }

  bool IsSpaceOrTab( int ch ) {
    return ch == ' ' || ch == '\t';
  }

  bool OnlySpaceOrTab( const std::string &s ) {
    for( std::string::const_iterator it = s.begin(); it != s.end(); ++it ) {
      if( !IsSpaceOrTab( *it ) ) {
        return false;
      }
    }
    return true;
  }

  std::vector<std::string> StringSplit( const std::string &text, int separator ) {
    std::vector<std::string> vs( text.empty() ? 0 : 1 );
    for( std::string::const_iterator it = text.begin(); it != text.end(); ++it ) {
      if( *it == separator ) {
        vs.push_back( std::string() );
      } else
      { vs.back() += *it; }
    }
    return vs;
  }

  struct BracketPair {
    std::vector<std::string>::iterator itBracket;
    std::vector<std::string>::iterator itEndBracket;
  };

  BracketPair FindBracketPair( std::vector<std::string> &tokens ) {
    BracketPair bp;
    std::vector<std::string>::iterator itTok = std::find( tokens.begin(), tokens.end(), "(" );
    bp.itBracket = tokens.end();
    bp.itEndBracket = tokens.end();
    if( itTok != tokens.end() ) {
      bp.itBracket = itTok;
      size_t nest = 0;
      while( itTok != tokens.end() ) {
        if( *itTok == "(" ) {
          nest++;
        } else if( *itTok == ")" ) {
          nest--;
          if( nest == 0 ) {
            bp.itEndBracket = itTok;
            return bp;
          }
        }
        ++itTok;
      }
    }
    bp.itBracket = tokens.end();
    return bp;
  }

  void highlightTaskMarker( StyleContext &sc, LexAccessor &styler,
                            int activity, WordList &markerList, bool caseSensitive ) {
    if( ( isoperator( sc.chPrev ) || IsASpace( sc.chPrev ) ) && markerList.Length() ) {
      const int lengthMarker = 50;
      char marker[lengthMarker + 1];
      int currPos = ( int ) sc.currentPos;
      int i = 0;
      while( i < lengthMarker ) {
        char ch = styler.SafeGetCharAt( currPos + i );
        if( IsASpace( ch ) || isoperator( ch ) ) {
          break;
        }
        if( caseSensitive ) {
          marker[i] = ch;
        } else
        { marker[i] = static_cast<char>( tolower( ch ) ); }
        i++;
      }
      marker[i] = '\0';
      if( markerList.InList( marker ) ) {
        sc.SetState( SCE_C_TASKMARKER | activity );
      }
    }
  }

  struct EscapeSequence {
    int digitsLeft;
    CharacterSet setHexDigits;
    CharacterSet setOctDigits;
    CharacterSet setNoneNumeric;
    CharacterSet *escapeSetValid;
    EscapeSequence() {
      digitsLeft = 0;
      escapeSetValid = 0;
      setHexDigits = CharacterSet( CharacterSet::setDigits, "ABCDEFabcdef" );
      setOctDigits = CharacterSet( CharacterSet::setNone, "01234567" );
    }
    void resetEscapeState( int nextChar ) {
      digitsLeft = 0;
      escapeSetValid = &setNoneNumeric;
      if( nextChar == 'U' ) {
        digitsLeft = 9;
        escapeSetValid = &setHexDigits;
      } else if( nextChar == 'u' ) {
        digitsLeft = 5;
        escapeSetValid = &setHexDigits;
      } else if( nextChar == 'x' ) {
        digitsLeft = 5;
        escapeSetValid = &setHexDigits;
      } else if( setOctDigits.Contains( nextChar ) ) {
        digitsLeft = 3;
        escapeSetValid = &setOctDigits;
      }
    }
    bool atEscapeEnd( int currChar ) const {
      return ( digitsLeft <= 0 ) || !escapeSetValid->Contains( currChar );
    }
  };

  std::string GetRestOfLine( LexAccessor &styler, int start, bool allowSpace ) {
    std::string restOfLine;
    int i = 0;
    char ch = styler.SafeGetCharAt( start, '\n' );
    int endLine = styler.LineEnd( styler.GetLine( start ) );
    while( ( ( start + i ) < endLine ) && ( ch != '\r' ) ) {
      char chNext = styler.SafeGetCharAt( start + i + 1, '\n' );
      if( ch == '/' && ( chNext == '/' || chNext == '*' ) ) {
        break;
      }
      if( allowSpace || ( ch != ' ' ) ) {
        restOfLine += ch;
      }
      i++;
      ch = chNext;
    }
    return restOfLine;
  }

  bool IsStreamCommentStyle( int style ) {
    return style == SCE_C_COMMENT ||
           style == SCE_C_COMMENTDOC ||
           style == SCE_C_COMMENTDOCKEYWORD ||
           style == SCE_C_COMMENTDOCKEYWORDERROR;
  }

  struct PPDefinition {
    int line;
    std::string key;
    std::string value;
    bool isUndef;
    std::string arguments;
    PPDefinition( int line_, const std::string &key_, const std::string &value_, bool isUndef_ = false, std::string arguments_ = "" ) :
      line( line_ ), key( key_ ), value( value_ ), isUndef( isUndef_ ), arguments( arguments_ ) {
    }
  };

  class LinePPState {
      int state;
      int ifTaken;
      int level;
      bool ValidLevel() const {
        return level >= 0 && level < 32;
      }
      int maskLevel() const {
        return 1 << level;
      }
    public:
      LinePPState() : state( 0 ), ifTaken( 0 ), level( -1 ) {
      }
      bool IsInactive() const {
        return state != 0;
      }
      bool CurrentIfTaken() const {
        return ( ifTaken & maskLevel() ) != 0;
      }
      void StartSection( bool on ) {
        level++;
        if( ValidLevel() ) {
          if( on ) {
            state &= ~maskLevel();
            ifTaken |= maskLevel();
          } else {
            state |= maskLevel();
            ifTaken &= ~maskLevel();
          }
        }
      }
      void EndSection() {
        if( ValidLevel() ) {
          state &= ~maskLevel();
          ifTaken &= ~maskLevel();
        }
        level--;
      }
      void InvertCurrentLevel() {
        if( ValidLevel() ) {
          state ^= maskLevel();
          ifTaken |= maskLevel();
        }
      }
  };

  class PPStates {
      std::vector<LinePPState> vlls;
    public:
      LinePPState ForLine( int line ) const {
        if( ( line > 0 ) && ( vlls.size() > static_cast<size_t>( line ) ) ) {
          return vlls[line];
        } else
        { return LinePPState(); }
      }
      void Add( int line, LinePPState lls ) {
        vlls.resize( line + 1 );
        vlls[line] = lls;
      }
  };

  struct OptionsCPP {
    bool stylingWithinPreprocessor;
    bool identifiersAllowDollars;
    bool trackPreprocessor;
    bool updatePreprocessor;
    bool triplequotedStrings;
    bool hashquotedStrings;
    bool backQuotedStrings;
    bool escapeSequence;
    bool fold;
    bool foldSyntaxBased;
    bool foldComment;
    bool foldCommentMultiline;
    bool foldCommentExplicit;
    std::string foldExplicitStart;
    std::string foldExplicitEnd;
    bool foldExplicitAnywhere;
    bool foldPreprocessor;
    bool foldCompact;
    bool foldAtElse;

    bool highlightWxSmith;

    OptionsCPP() {
      stylingWithinPreprocessor = false;
      identifiersAllowDollars = true;
      trackPreprocessor = true;
      updatePreprocessor = true;
      triplequotedStrings = false;
      hashquotedStrings = false;
      backQuotedStrings = false;
      escapeSequence = false;
      fold = false;
      foldSyntaxBased = true;
      foldComment = false;
      foldCommentMultiline = true;
      foldCommentExplicit = true;
      foldExplicitStart = "";
      foldExplicitEnd = "";
      foldExplicitAnywhere = false;
      foldPreprocessor = false;
      foldCompact = false;
      foldAtElse = false;
      highlightWxSmith = true;
    }
  };

  const char *const cppWordLists[] = {
    "Primary keywords and identifiers",
    "Secondary keywords and identifiers",
    "Documentation comment keywords",
    "Global classes and typedefs",
    "Preprocessor definitions",
    "Task marker and error marker keywords",

    "wxSmith block identifiers",

    0,
  };

  struct OptionSetCPP : public OptionSet<OptionsCPP> {
    OptionSetCPP() {
      DefineProperty( "styling.within.preprocessor", &OptionsCPP::stylingWithinPreprocessor,
                      "For C++ code, determines whether all preprocessor code is styled in the "
                      "preprocessor style (0, the default) or only from the initial # to the end "
                      "of the command word(1)." );
      DefineProperty( "lexer.cpp.allow.dollars", &OptionsCPP::identifiersAllowDollars,
                      "Set to 0 to disallow the '$' character in identifiers with the cpp lexer." );
      DefineProperty( "lexer.cpp.track.preprocessor", &OptionsCPP::trackPreprocessor,
                      "Set to 1 to interpret #if/#else/#endif to grey out code that is not active." );
      DefineProperty( "lexer.cpp.update.preprocessor", &OptionsCPP::updatePreprocessor,
                      "Set to 1 to update preprocessor definitions when #define found." );
      DefineProperty( "lexer.cpp.triplequoted.strings", &OptionsCPP::triplequotedStrings,
                      "Set to 1 to enable highlighting of triple-quoted strings." );
      DefineProperty( "lexer.cpp.hashquoted.strings", &OptionsCPP::hashquotedStrings,
                      "Set to 1 to enable highlighting of hash-quoted strings." );
      DefineProperty( "lexer.cpp.backquoted.strings", &OptionsCPP::backQuotedStrings,
                      "Set to 1 to enable highlighting of back-quoted raw strings ." );
      DefineProperty( "lexer.cpp.escape.sequence", &OptionsCPP::escapeSequence,
                      "Set to 1 to enable highlighting of escape sequences in strings" );
      DefineProperty( "fold", &OptionsCPP::fold );
      DefineProperty( "fold.cpp.syntax.based", &OptionsCPP::foldSyntaxBased,
                      "Set this property to 0 to disable syntax based folding." );
      DefineProperty( "fold.comment", &OptionsCPP::foldComment,
                      "This option enables folding multi-line comments and explicit fold points when using the C++ lexer. "
                      "Explicit fold points allows adding extra folding by placing a //{ comment at the start and a //} "
                      "at the end of a section that should fold." );
      DefineProperty( "fold.cpp.comment.multiline", &OptionsCPP::foldCommentMultiline,
                      "Set this property to 0 to disable folding multi-line comments when fold.comment=1." );
      DefineProperty( "fold.cpp.comment.explicit", &OptionsCPP::foldCommentExplicit,
                      "Set this property to 0 to disable folding explicit fold points when fold.comment=1." );
      DefineProperty( "fold.cpp.explicit.start", &OptionsCPP::foldExplicitStart,
                      "The string to use for explicit fold start points, replacing the standard //{." );
      DefineProperty( "fold.cpp.explicit.end", &OptionsCPP::foldExplicitEnd,
                      "The string to use for explicit fold end points, replacing the standard //}." );
      DefineProperty( "fold.cpp.explicit.anywhere", &OptionsCPP::foldExplicitAnywhere,
                      "Set this property to 1 to enable explicit fold points anywhere, not just in line comments." );
      DefineProperty( "fold.preprocessor", &OptionsCPP::foldPreprocessor,
                      "This option enables folding preprocessor directives when using the C++ lexer. "
                      "Includes C#'s explicit #region and #endregion folding directives." );
      DefineProperty( "fold.compact", &OptionsCPP::foldCompact );
      DefineProperty( "highlight.wxsmith", &OptionsCPP::highlightWxSmith );
      DefineProperty( "fold.at.else", &OptionsCPP::foldAtElse,
                      "This option enables C++ folding on a \"} else {\" line of an if statement." );
      DefineWordListSets( cppWordLists );
    }
  };

  const char styleSubable[] = {SCE_C_IDENTIFIER, SCE_C_COMMENTDOCKEYWORD, 0};

}

class LexerCPP : public ILexerWithSubStyles {
    bool caseSensitive;
    CharacterSet setWord;
    CharacterSet setNegationOp;
    CharacterSet setArithmethicOp;
    CharacterSet setRelOp;
    CharacterSet setLogicalOp;
    CharacterSet setWordStart;
    PPStates vlls;
    std::vector<PPDefinition> ppDefineHistory;
    WordList keywords;
    WordList keywords2;
    WordList keywords3;
    WordList keywords4;
    WordList ppDefinitions;
    WordList markerList;

    WordList wxSmithIds;

    struct SymbolValue {
      std::string value;
      std::string arguments;
      SymbolValue( const std::string &value_ = "", const std::string &arguments_ = "" ) : value( value_ ), arguments( arguments_ ) {
      }
      SymbolValue &operator = ( const std::string &value_ ) {
        value = value_;
        arguments.clear();
        return *this;
      }
      bool IsMacro() const {
        return !arguments.empty();
      }
    };
    typedef std::map<std::string, SymbolValue> SymbolTable;
    SymbolTable preprocessorDefinitionsStart;
    OptionsCPP options;
    OptionSetCPP osCPP;
    EscapeSequence escapeSeq;
    SparseState<std::string> rawStringTerminators;
    enum { activeFlag = 0x40 };
    enum { ssIdentifier, ssDocKeyword };
    SubStyles subStyles;
  public:
    explicit LexerCPP( bool caseSensitive_ ) :
      caseSensitive( caseSensitive_ ),
      setWord( CharacterSet::setAlphaNum, "._", 0x80, true ),
      setNegationOp( CharacterSet::setNone, "!" ),
      setArithmethicOp( CharacterSet::setNone, "+-/*%" ),
      setRelOp( CharacterSet::setNone, "=!<>" ),
      setLogicalOp( CharacterSet::setNone, "|&" ),
      subStyles( styleSubable, 0x80, 0x40, activeFlag ) {
    }
    virtual ~LexerCPP() {
    }
    void  Release() {
      delete this;
    }
    int  Version() const {
      return lvSubStyles;
    }
    const char *  PropertyNames() {
      return osCPP.PropertyNames();
    }
    int  PropertyType( const char *name ) {
      return osCPP.PropertyType( name );
    }
    const char *  DescribeProperty( const char *name ) {
      return osCPP.DescribeProperty( name );
    }
    int  PropertySet( const char *key, const char *val );
    const char *  DescribeWordListSets() {
      return osCPP.DescribeWordListSets();
    }
    int  WordListSet( int n, const char *wl );
    void  Lex( unsigned int startPos, int length, int initStyle, IDocument *pAccess );
    void  Fold( unsigned int startPos, int length, int initStyle, IDocument *pAccess );

    void *  PrivateCall( int, void * ) {
      return 0;
    }

    int  LineEndTypesSupported() {
      return SC_LINE_END_TYPE_UNICODE;
    }

    int  AllocateSubStyles( int styleBase, int numberStyles ) {
      return subStyles.Allocate( styleBase, numberStyles );
    }
    int  SubStylesStart( int styleBase ) {
      return subStyles.Start( styleBase );
    }
    int  SubStylesLength( int styleBase ) {
      return subStyles.Length( styleBase );
    }
    int  StyleFromSubStyle( int subStyle ) {
      int styleBase = subStyles.BaseStyle( MaskActive( subStyle ) );
      int active = subStyle & activeFlag;
      return styleBase | active;
    }
    int  PrimaryStyleFromStyle( int style ) {
      return MaskActive( style );
    }
    void  FreeSubStyles() {
      subStyles.Free();
    }
    void  SetIdentifiers( int style, const char *identifiers ) {
      subStyles.SetIdentifiers( style, identifiers );
    }
    int  DistanceToSecondaryStyles() {
      return activeFlag;
    }
    const char *  GetSubStyleBases() {
      return styleSubable;
    }

    static ILexer *LexerFactoryCPP() {
      return new LexerCPP( true );
    }
    static ILexer *LexerFactoryCPPInsensitive() {
      return new LexerCPP( false );
    }
    static int MaskActive( int style ) {
      return style & ~activeFlag;
    }
    void EvaluateTokens( std::vector<std::string> &tokens, const SymbolTable &preprocessorDefinitions );
    std::vector<std::string> Tokenize( const std::string &expr ) const;
    bool EvaluateExpression( const std::string &expr, const SymbolTable &preprocessorDefinitions );
};

int  LexerCPP::PropertySet( const char *key, const char *val ) {
  if( osCPP.PropertySet( &options, key, val ) ) {
    if( strcmp( key, "lexer.cpp.allow.dollars" ) == 0 ) {
      setWord = CharacterSet( CharacterSet::setAlphaNum, "._", 0x80, true );
      if( options.identifiersAllowDollars ) {
        setWord.Add( '$' );
      }
    }
    return 0;
  }
  return -1;
}

int  LexerCPP::WordListSet( int n, const char *wl ) {
  WordList *wordListN = 0;
  switch( n ) {
    case 0:
      wordListN = &keywords;
      break;
    case 1:
      wordListN = &keywords2;
      break;
    case 2:
      wordListN = &keywords3;
      break;
    case 3:
      wordListN = &keywords4;
      break;
    case 4:
      wordListN = &ppDefinitions;
      break;
    case 5:
      wordListN = &markerList;
      break;
    case 6:
      wordListN = &wxSmithIds;
      break;
  }
  int firstModification = -1;
  if( wordListN ) {
    WordList wlNew;
    wlNew.Set( wl );
    if( *wordListN != wlNew ) {
      wordListN->Set( wl );
      firstModification = 0;
      if( n == 4 ) {
        preprocessorDefinitionsStart.clear();
        for( int nDefinition = 0; nDefinition < ppDefinitions.Length(); nDefinition++ ) {
          const char *cpDefinition = ppDefinitions.WordAt( nDefinition );
          const char *cpEquals = strchr( cpDefinition, '=' );
          if( cpEquals ) {
            std::string name( cpDefinition, cpEquals - cpDefinition );
            std::string val( cpEquals + 1 );
            size_t bracket = name.find( '(' );
            size_t bracketEnd = name.find( ')' );
            if( ( bracket != std::string::npos ) && ( bracketEnd != std::string::npos ) ) {
              std::string args = name.substr( bracket + 1, bracketEnd - bracket - 1 );
              name = name.substr( 0, bracket );
              preprocessorDefinitionsStart[name] = SymbolValue( val, args );
            } else
            { preprocessorDefinitionsStart[name] = val; }
          } else {
            std::string name( cpDefinition );
            std::string val( "1" );
            preprocessorDefinitionsStart[name] = val;
          }
        }
      }
    }
  }
  return firstModification;
}

struct After {
  int line;
  explicit After( int line_ ) : line( line_ ) {}
  bool operator()( PPDefinition &p ) const {
    return p.line > line;
  }
};

void  LexerCPP::Lex( unsigned int startPos, int length, int initStyle, IDocument *pAccess ) {
  LexAccessor styler( pAccess );
  CharacterSet setOKBeforeRE( CharacterSet::setNone, "([{=,:;!%^&*|?~+-" );
  CharacterSet setCouldBePostOp( CharacterSet::setNone, "+-" );
  CharacterSet setDoxygen( CharacterSet::setAlpha, "$@\\&<>#{}[]" );
  setWordStart = CharacterSet( CharacterSet::setAlpha, "_", 0x80, true );
  CharacterSet setInvalidRawFirst( CharacterSet::setNone, " )\\\t\v\f\n" );
  if( options.identifiersAllowDollars ) {
    setWordStart.Add( '$' );
  }
  int chPrevNonWhite = ' ';
  int visibleChars = 0;
  bool lastWordWasUUID = false;
  int styleBeforeDCKeyword = SCE_C_DEFAULT;
  int styleBeforeTaskMarker = SCE_C_DEFAULT;
  bool continuationLine = false;
  bool isIncludePreprocessor = false;
  bool isStringInPreprocessor = false;
  bool inRERange = false;
  bool seenDocKeyBrace = false;
  int lineCurrent = styler.GetLine( startPos );
  if( ( MaskActive( initStyle ) == SCE_C_PREPROCESSOR ) ||
      ( MaskActive( initStyle ) == SCE_C_COMMENTLINE ) ||
      ( MaskActive( initStyle ) == SCE_C_COMMENTLINEDOC ) ) {
    if( lineCurrent > 0 ) {
      int endLinePrevious = styler.LineEnd( lineCurrent - 1 );
      if( endLinePrevious > 0 ) {
        continuationLine = styler.SafeGetCharAt( endLinePrevious - 1 ) == '\\';
      }
    }
  }
  if( startPos > 0 ) {
    int back = startPos;
    while( --back && IsSpaceEquiv( MaskActive( styler.StyleAt( back ) ) ) )
      ;
    if( MaskActive( styler.StyleAt( back ) ) == SCE_C_OPERATOR ) {
      chPrevNonWhite = styler.SafeGetCharAt( back );
    }
  }
  StyleContext sc( startPos, length, initStyle, styler, static_cast<unsigned char>( 0xff ) );
  LinePPState preproc = vlls.ForLine( lineCurrent );
  bool definitionsChanged = false;
  if( !options.updatePreprocessor ) {
    ppDefineHistory.clear();
  }
  std::vector<PPDefinition>::iterator itInvalid = std::find_if( ppDefineHistory.begin(), ppDefineHistory.end(), After( lineCurrent - 1 ) );
  if( itInvalid != ppDefineHistory.end() ) {
    ppDefineHistory.erase( itInvalid, ppDefineHistory.end() );
    definitionsChanged = true;
  }
  SymbolTable preprocessorDefinitions = preprocessorDefinitionsStart;
  for( std::vector<PPDefinition>::iterator itDef = ppDefineHistory.begin(); itDef != ppDefineHistory.end(); ++itDef ) {
    if( itDef->isUndef ) {
      preprocessorDefinitions.erase( itDef->key );
    } else
    { preprocessorDefinitions[itDef->key] = SymbolValue( itDef->value, itDef->arguments ); }
  }
  std::string rawStringTerminator = rawStringTerminators.ValueAt( lineCurrent - 1 );
  SparseState<std::string> rawSTNew( lineCurrent );
  int activitySet = preproc.IsInactive() ? activeFlag : 0;
  const WordClassifier &classifierIdentifiers = subStyles.Classifier( SCE_C_IDENTIFIER );
  const WordClassifier &classifierDocKeyWords = subStyles.Classifier( SCE_C_COMMENTDOCKEYWORD );
  int lineEndNext = styler.LineEnd( lineCurrent );
  for( ; sc.More(); ) {
    if( sc.atLineStart ) {
      if( ( sc.state == SCE_C_STRING ) || ( sc.state == SCE_C_CHARACTER ) ) {
        sc.SetState( sc.state );
      }
      if( ( MaskActive( sc.state ) == SCE_C_PREPROCESSOR ) && ( !continuationLine ) ) {
        sc.SetState( SCE_C_DEFAULT | activitySet );
      }
      visibleChars = 0;
      lastWordWasUUID = false;
      isIncludePreprocessor = false;
      inRERange = false;
      if( preproc.IsInactive() ) {
        activitySet = activeFlag;
        sc.SetState( sc.state | activitySet );
      }
    }
    if( sc.atLineEnd ) {
      lineCurrent++;
      lineEndNext = styler.LineEnd( lineCurrent );
      vlls.Add( lineCurrent, preproc );
      if( rawStringTerminator != "" ) {
        rawSTNew.Set( lineCurrent - 1, rawStringTerminator );
      }
    }
    if( sc.ch == '\\' ) {
      if( static_cast<int>( ( sc.currentPos + 1 ) ) >= lineEndNext ) {
        lineCurrent++;
        lineEndNext = styler.LineEnd( lineCurrent );
        vlls.Add( lineCurrent, preproc );
        sc.Forward();
        if( sc.ch == '\r' && sc.chNext == '\n' ) {
          sc.Forward();
        }
        continuationLine = true;
        sc.Forward();
        continue;
      }
    }
    const bool atLineEndBeforeSwitch = sc.atLineEnd;
    switch( MaskActive( sc.state ) ) {
      case SCE_C_OPERATOR:
        sc.SetState( SCE_C_DEFAULT | activitySet );
        break;
      case SCE_C_NUMBER:
        if( sc.ch == '_' ) {
          sc.ChangeState( SCE_C_USERLITERAL | activitySet );
        } else if( !( setWord.Contains( sc.ch )
                      || ( sc.ch == '\'' )
                      || ( ( sc.ch == '+' || sc.ch == '-' ) && ( sc.chPrev == 'e' || sc.chPrev == 'E' ||
                           sc.chPrev == 'p' || sc.chPrev == 'P' ) ) ) ) {
          sc.SetState( SCE_C_DEFAULT | activitySet );
        }
        break;
      case SCE_C_USERLITERAL:
        if( !( setWord.Contains( sc.ch ) ) ) {
          sc.SetState( SCE_C_DEFAULT | activitySet );
        }
        break;
      case SCE_C_IDENTIFIER:
        if( sc.atLineStart || sc.atLineEnd || !setWord.Contains( sc.ch ) || ( sc.ch == '.' ) ) {
          char s[1000];
          if( caseSensitive ) {
            sc.GetCurrent( s, sizeof( s ) );
          } else
          { sc.GetCurrentLowered( s, sizeof( s ) ); }
          if( keywords.InList( s ) ) {
            lastWordWasUUID = strcmp( s, "uuid" ) == 0;
            sc.ChangeState( SCE_C_WORD | activitySet );
          } else if( keywords2.InList( s ) ) {
            sc.ChangeState( SCE_C_WORD2 | activitySet );
          } else if( keywords4.InList( s ) ) {
            sc.ChangeState( SCE_C_GLOBALCLASS | activitySet );
          } else {
            int subStyle = classifierIdentifiers.ValueFor( s );
            if( subStyle >= 0 )
            { sc.ChangeState( subStyle | activitySet ); }
          }
          const bool literalString = sc.ch == '\"';
          if( literalString || sc.ch == '\'' ) {
            size_t lenS = strlen( s );
            const bool raw = literalString && sc.chPrev == 'R' && !setInvalidRawFirst.Contains( sc.chNext );
            if( raw )
            { s[lenS--] = '\0'; }
            bool valid =
              ( lenS == 0 ) ||
              ( ( lenS == 1 ) && ( ( s[0] == 'L' ) || ( s[0] == 'u' ) || ( s[0] == 'U' ) ) ) ||
              ( ( lenS == 2 ) && literalString && ( s[0] == 'u' ) && ( s[1] == '8' ) );
            if( valid ) {
              if( literalString ) {
                if( raw ) {
                  sc.ChangeState( SCE_C_STRINGRAW | activitySet );
                  sc.SetState( SCE_C_DEFAULT | activitySet );
                } else
                { sc.ChangeState( SCE_C_STRING | activitySet ); }
              } else
              { sc.ChangeState( SCE_C_CHARACTER | activitySet ); }
            } else
            { sc.SetState( SCE_C_DEFAULT | activitySet ); }
          } else
          { sc.SetState( SCE_C_DEFAULT | activitySet ); }
        }
        break;
      case SCE_C_PREPROCESSOR:
        if( options.stylingWithinPreprocessor ) {
          if( IsASpace( sc.ch ) ) {
            sc.SetState( SCE_C_DEFAULT | activitySet );
          }
        } else if( isStringInPreprocessor && ( sc.Match( '>' ) || sc.Match( '\"' ) || sc.atLineEnd ) ) {
          isStringInPreprocessor = false;
        } else if( !isStringInPreprocessor ) {
          if( ( isIncludePreprocessor && sc.Match( '<' ) ) || sc.Match( '\"' ) ) {
            isStringInPreprocessor = true;
          } else if( sc.Match( '/', '*' ) ) {
            if( sc.Match( "/**" ) || sc.Match( "/*!" ) )
            { sc.SetState( SCE_C_PREPROCESSORCOMMENTDOC | activitySet ); }
            else
            { sc.SetState( SCE_C_PREPROCESSORCOMMENT | activitySet ); }
            sc.Forward();
          } else if( sc.Match( '/', '/' ) ) {
            sc.SetState( SCE_C_DEFAULT | activitySet );
          }
        }
        break;
      case SCE_C_PREPROCESSORCOMMENT:
      case SCE_C_PREPROCESSORCOMMENTDOC:
        if( sc.Match( '*', '/' ) ) {
          sc.Forward();
          sc.ForwardSetState( SCE_C_PREPROCESSOR | activitySet );
          continue;
        }
        break;
      case SCE_C_COMMENT:
        if( sc.Match( '*', '/' ) ) {
          sc.Forward();
          sc.ForwardSetState( SCE_C_DEFAULT | activitySet );
        } else {
          styleBeforeTaskMarker = SCE_C_COMMENT;
          highlightTaskMarker( sc, styler, activitySet, markerList, caseSensitive );
        }
        break;
      case SCE_C_COMMENTDOC:
        if( sc.Match( '*', '/' ) ) {
          sc.Forward();
          sc.ForwardSetState( SCE_C_DEFAULT | activitySet );
        } else if( sc.ch == '@' || sc.ch == '\\' ) {
          if( ( IsASpace( sc.chPrev ) || sc.chPrev == '*' ) && ( !IsASpace( sc.chNext ) ) ) {
            styleBeforeDCKeyword = SCE_C_COMMENTDOC;
            sc.SetState( SCE_C_COMMENTDOCKEYWORD | activitySet );
          }
        }
        break;
      case SCE_C_COMMENTLINE:
        if( sc.atLineStart && !continuationLine ) {
          sc.SetState( SCE_C_DEFAULT | activitySet );
        } else {
          styleBeforeTaskMarker = SCE_C_COMMENTLINE;
          highlightTaskMarker( sc, styler, activitySet, markerList, caseSensitive );
        }
        break;
      case SCE_C_COMMENTLINEDOC:
        if( sc.atLineStart && !continuationLine ) {
          sc.SetState( SCE_C_DEFAULT | activitySet );
        } else if( sc.ch == '@' || sc.ch == '\\' ) {
          if( ( IsASpace( sc.chPrev ) || sc.chPrev == '/' || sc.chPrev == '!' ) && ( !IsASpace( sc.chNext ) ) ) {
            styleBeforeDCKeyword = SCE_C_COMMENTLINEDOC;
            sc.SetState( SCE_C_COMMENTDOCKEYWORD | activitySet );
          }
        }
        break;
      case SCE_C_COMMENTDOCKEYWORD:
        if( ( styleBeforeDCKeyword == SCE_C_COMMENTDOC ) && sc.Match( '*', '/' ) ) {
          sc.ChangeState( SCE_C_COMMENTDOCKEYWORDERROR );
          sc.Forward();
          sc.ForwardSetState( SCE_C_DEFAULT | activitySet );
          seenDocKeyBrace = false;
        } else if( sc.ch == '[' || sc.ch == '{' ) {
          seenDocKeyBrace = true;
        } else if( !setDoxygen.Contains( sc.ch )
                   && !( seenDocKeyBrace && ( sc.ch == ',' || sc.ch == '.' ) ) ) {
          char s[100];
          if( caseSensitive ) {
            sc.GetCurrent( s, sizeof( s ) );
          } else
          { sc.GetCurrentLowered( s, sizeof( s ) ); }
          if( !( IsASpace( sc.ch ) || ( sc.ch == 0 ) ) ) {
            sc.ChangeState( SCE_C_COMMENTDOCKEYWORDERROR | activitySet );
          } else if( !keywords3.InList( s + 1 ) ) {
            int subStyleCDKW = classifierDocKeyWords.ValueFor( s + 1 );
            if( subStyleCDKW >= 0 )
            { sc.ChangeState( subStyleCDKW | activitySet ); }
            else
            { sc.ChangeState( SCE_C_COMMENTDOCKEYWORDERROR | activitySet ); }
          }
          sc.SetState( styleBeforeDCKeyword | activitySet );
          seenDocKeyBrace = false;
        }
        break;
      case SCE_C_STRING:
        if( sc.atLineEnd ) {
          sc.ChangeState( SCE_C_STRINGEOL | activitySet );
        } else if( isIncludePreprocessor ) {
          if( sc.ch == '>' ) {
            sc.ForwardSetState( SCE_C_DEFAULT | activitySet );
            isIncludePreprocessor = false;
          }
        } else if( sc.ch == '\\' ) {
          if( options.escapeSequence ) {
            sc.SetState( SCE_C_ESCAPESEQUENCE | activitySet );
            escapeSeq.resetEscapeState( sc.chNext );
          }
          sc.Forward();
        } else if( sc.ch == '\"' ) {
          if( sc.chNext == '_' ) {
            sc.ChangeState( SCE_C_USERLITERAL | activitySet );
          } else
          { sc.ForwardSetState( SCE_C_DEFAULT | activitySet ); }
        }
        break;
      case SCE_C_ESCAPESEQUENCE:
        escapeSeq.digitsLeft--;
        if( !escapeSeq.atEscapeEnd( sc.ch ) ) {
          break;
        }
        if( sc.ch == '"' ) {
          sc.SetState( SCE_C_STRING | activitySet );
          sc.ForwardSetState( SCE_C_DEFAULT | activitySet );
        } else if( sc.ch == '\\' ) {
          escapeSeq.resetEscapeState( sc.chNext );
          sc.Forward();
        } else {
          sc.SetState( SCE_C_STRING | activitySet );
          if( sc.atLineEnd ) {
            sc.ChangeState( SCE_C_STRINGEOL | activitySet );
          }
        }
        break;
      case SCE_C_HASHQUOTEDSTRING:
        if( sc.ch == '\\' ) {
          if( sc.chNext == '\"' || sc.chNext == '\'' || sc.chNext == '\\' ) {
            sc.Forward();
          }
        } else if( sc.ch == '\"' ) {
          sc.ForwardSetState( SCE_C_DEFAULT | activitySet );
        }
        break;
      case SCE_C_STRINGRAW:
        if( sc.Match( rawStringTerminator.c_str() ) ) {
          for( size_t termPos = rawStringTerminator.size(); termPos; termPos-- ) {
            sc.Forward();
          }
          sc.SetState( SCE_C_DEFAULT | activitySet );
          rawStringTerminator = "";
        }
        break;
      case SCE_C_CHARACTER:
        if( sc.atLineEnd ) {
          sc.ChangeState( SCE_C_STRINGEOL | activitySet );
        } else if( sc.ch == '\\' ) {
          if( sc.chNext == '\"' || sc.chNext == '\'' || sc.chNext == '\\' ) {
            sc.Forward();
          }
        } else if( sc.ch == '\'' ) {
          if( sc.chNext == '_' ) {
            sc.ChangeState( SCE_C_USERLITERAL | activitySet );
          } else
          { sc.ForwardSetState( SCE_C_DEFAULT | activitySet ); }
        }
        break;
      case SCE_C_REGEX:
        if( sc.atLineStart ) {
          sc.SetState( SCE_C_DEFAULT | activitySet );
        } else if( ! inRERange && sc.ch == '/' ) {
          sc.Forward();
          while( ( sc.ch < 0x80 ) && islower( sc.ch ) ) {
            sc.Forward();
          }
          sc.SetState( SCE_C_DEFAULT | activitySet );
        } else if( sc.ch == '\\' && ( static_cast<int>( sc.currentPos + 1 ) < lineEndNext ) ) {
          sc.Forward();
        } else if( sc.ch == '[' ) {
          inRERange = true;
        } else if( sc.ch == ']' ) {
          inRERange = false;
        }
        break;
      case SCE_C_STRINGEOL:
        if( sc.atLineStart ) {
          sc.SetState( SCE_C_DEFAULT | activitySet );
        }
        break;
      case SCE_C_VERBATIM:
        if( sc.ch == '\"' ) {
          if( sc.chNext == '\"' ) {
            sc.Forward();
          } else
          { sc.ForwardSetState( SCE_C_DEFAULT | activitySet ); }
        }
        break;
      case SCE_C_TRIPLEVERBATIM:
        if( sc.Match( "\"\"\"" ) ) {
          while( sc.Match( '"' ) ) {
            sc.Forward();
          }
          sc.SetState( SCE_C_DEFAULT | activitySet );
        }
        break;
      case SCE_C_UUID:
        if( sc.atLineEnd || sc.ch == ')' ) {
          sc.SetState( SCE_C_DEFAULT | activitySet );
        }
        break;
      case SCE_C_TASKMARKER:
        if( isoperator( sc.ch ) || IsASpace( sc.ch ) ) {
          sc.SetState( styleBeforeTaskMarker | activitySet );
          styleBeforeTaskMarker = SCE_C_DEFAULT;
        }
        break;
      case SCE_C_WXSMITH:
        if( sc.Match( "//*)" ) ) {
          sc.Forward( 4 );
          sc.SetState( SCE_C_COMMENTLINE | activitySet );
          continue;
        }
    }
    if( sc.atLineEnd && !atLineEndBeforeSwitch ) {
      lineCurrent++;
      lineEndNext = styler.LineEnd( lineCurrent );
      vlls.Add( lineCurrent, preproc );
    }
    if( MaskActive( sc.state ) == SCE_C_DEFAULT ) {
      if( sc.Match( '@', '\"' ) ) {
        sc.SetState( SCE_C_VERBATIM | activitySet );
        sc.Forward();
      } else if( options.triplequotedStrings && sc.Match( "\"\"\"" ) ) {
        sc.SetState( SCE_C_TRIPLEVERBATIM | activitySet );
        sc.Forward( 2 );
      } else if( options.hashquotedStrings && sc.Match( '#', '\"' ) ) {
        sc.SetState( SCE_C_HASHQUOTEDSTRING | activitySet );
        sc.Forward();
      } else if( options.backQuotedStrings && sc.Match( '`' ) ) {
        sc.SetState( SCE_C_STRINGRAW | activitySet );
        rawStringTerminator = "`";
        sc.Forward();
      } else if( IsADigit( sc.ch ) || ( sc.ch == '.' && IsADigit( sc.chNext ) ) ) {
        if( lastWordWasUUID ) {
          sc.SetState( SCE_C_UUID | activitySet );
          lastWordWasUUID = false;
        } else
        { sc.SetState( SCE_C_NUMBER | activitySet ); }
      } else if( !sc.atLineEnd && ( setWordStart.Contains( sc.ch ) || ( sc.ch == '@' ) ) ) {
        if( lastWordWasUUID ) {
          sc.SetState( SCE_C_UUID | activitySet );
          lastWordWasUUID = false;
        } else
        { sc.SetState( SCE_C_IDENTIFIER | activitySet ); }
      } else if( sc.Match( '/', '*' ) ) {
        if( sc.Match( "/**" ) || sc.Match( "/*!" ) ) {
          sc.SetState( SCE_C_COMMENTDOC | activitySet );
        } else
        { sc.SetState( SCE_C_COMMENT | activitySet ); }
        sc.Forward();
      } else if( sc.Match( '/', '/' ) ) {
        if( ( sc.Match( "///" ) && !sc.Match( "////" ) ) || sc.Match( "//!" ) ) {
          sc.SetState( SCE_C_COMMENTLINEDOC | activitySet );
        } else if( sc.Match( "//(*" ) && options.highlightWxSmith && wxSmithIds.InList( GetRestOfLine( styler, sc.currentPos + 4, true ).c_str() ) ) {
          sc.SetState( SCE_C_WXSMITH | activitySet );
          sc.Forward( 4 );
          sc.SetState( SCE_C_COMMENTLINE | activitySet );
          while( !sc.atLineEnd ) {
            sc.Forward();
          }
          sc.SetState( SCE_C_WXSMITH | activitySet );
        } else
        { sc.SetState( SCE_C_COMMENTLINE | activitySet ); }
      } else if( sc.ch == '/'
                 && ( setOKBeforeRE.Contains( chPrevNonWhite )
                      || followsReturnKeyword( sc, styler ) )
                 && ( !setCouldBePostOp.Contains( chPrevNonWhite )
                      || !FollowsPostfixOperator( sc, styler ) ) ) {
        sc.SetState( SCE_C_REGEX | activitySet );
        inRERange = false;
      } else if( sc.ch == '\"' ) {
        if( sc.chPrev == 'R' ) {
          styler.Flush();
          if( MaskActive( styler.StyleAt( sc.currentPos - 1 ) ) == SCE_C_STRINGRAW ) {
            sc.SetState( SCE_C_STRINGRAW | activitySet );
            rawStringTerminator = ")";
            for( int termPos = sc.currentPos + 1;; termPos++ ) {
              char chTerminator = styler.SafeGetCharAt( termPos, '(' );
              if( chTerminator == '(' )
              { break; }
              rawStringTerminator += chTerminator;
            }
            rawStringTerminator += '\"';
          } else
          { sc.SetState( SCE_C_STRING | activitySet ); }
        } else
        { sc.SetState( SCE_C_STRING | activitySet ); }
        isIncludePreprocessor = false;
      } else if( isIncludePreprocessor && sc.ch == '<' ) {
        sc.SetState( SCE_C_STRING | activitySet );
      } else if( sc.ch == '\'' ) {
        sc.SetState( SCE_C_CHARACTER | activitySet );
      } else if( sc.ch == '#' && visibleChars == 0 ) {
        sc.SetState( SCE_C_PREPROCESSOR | activitySet );
        do {
          sc.Forward();
        } while( ( sc.ch == ' ' || sc.ch == '\t' ) && sc.More() );
        if( sc.atLineEnd ) {
          sc.SetState( SCE_C_DEFAULT | activitySet );
        } else if( sc.Match( "include" ) ) {
          isIncludePreprocessor = true;
        } else {
          if( options.trackPreprocessor ) {
            if( sc.Match( "ifdef" ) || sc.Match( "ifndef" ) ) {
              bool isIfDef = sc.Match( "ifdef" );
              int i = isIfDef ? 5 : 6;
              std::string restOfLine = GetRestOfLine( styler, sc.currentPos + i + 1, false );
              bool foundDef = preprocessorDefinitions.find( restOfLine ) != preprocessorDefinitions.end();
              preproc.StartSection( isIfDef == foundDef );
            } else if( sc.Match( "if" ) ) {
              std::string restOfLine = GetRestOfLine( styler, sc.currentPos + 2, true );
              bool ifGood = EvaluateExpression( restOfLine, preprocessorDefinitions );
              preproc.StartSection( ifGood );
            } else if( sc.Match( "else" ) ) {
              if( !preproc.CurrentIfTaken() ) {
                preproc.InvertCurrentLevel();
                activitySet = preproc.IsInactive() ? activeFlag : 0;
                if( !activitySet )
                { sc.ChangeState( SCE_C_PREPROCESSOR | activitySet ); }
              } else if( !preproc.IsInactive() ) {
                preproc.InvertCurrentLevel();
                activitySet = preproc.IsInactive() ? activeFlag : 0;
                if( !activitySet )
                { sc.ChangeState( SCE_C_PREPROCESSOR | activitySet ); }
              }
            } else if( sc.Match( "elif" ) ) {
              if( !preproc.CurrentIfTaken() ) {
                std::string restOfLine = GetRestOfLine( styler, sc.currentPos + 2, true );
                bool ifGood = EvaluateExpression( restOfLine, preprocessorDefinitions );
                if( ifGood ) {
                  preproc.InvertCurrentLevel();
                  activitySet = preproc.IsInactive() ? activeFlag : 0;
                  if( !activitySet )
                  { sc.ChangeState( SCE_C_PREPROCESSOR | activitySet ); }
                }
              } else if( !preproc.IsInactive() ) {
                preproc.InvertCurrentLevel();
                activitySet = preproc.IsInactive() ? activeFlag : 0;
                if( !activitySet )
                { sc.ChangeState( SCE_C_PREPROCESSOR | activitySet ); }
              }
            } else if( sc.Match( "endif" ) ) {
              preproc.EndSection();
              activitySet = preproc.IsInactive() ? activeFlag : 0;
              sc.ChangeState( SCE_C_PREPROCESSOR | activitySet );
            } else if( sc.Match( "define" ) ) {
              if( options.updatePreprocessor && !preproc.IsInactive() ) {
                std::string restOfLine = GetRestOfLine( styler, sc.currentPos + 6, true );
                size_t startName = 0;
                while( ( startName < restOfLine.length() ) && IsSpaceOrTab( restOfLine[startName] ) )
                { startName++; }
                size_t endName = startName;
                while( ( endName < restOfLine.length() ) && setWord.Contains( static_cast<unsigned char>( restOfLine[endName] ) ) )
                { endName++; }
                std::string key = restOfLine.substr( startName, endName - startName );
                if( ( endName < restOfLine.length() ) && ( restOfLine.at( endName ) == '(' ) ) {
                  size_t endArgs = endName;
                  while( ( endArgs < restOfLine.length() ) && ( restOfLine[endArgs] != ')' ) )
                  { endArgs++; }
                  std::string args = restOfLine.substr( endName + 1, endArgs - endName - 1 );
                  size_t startValue = endArgs + 1;
                  while( ( startValue < restOfLine.length() ) && IsSpaceOrTab( restOfLine[startValue] ) )
                  { startValue++; }
                  std::string value;
                  if( startValue < restOfLine.length() )
                  { value = restOfLine.substr( startValue ); }
                  preprocessorDefinitions[key] = SymbolValue( value, args );
                  ppDefineHistory.push_back( PPDefinition( lineCurrent, key, value, false, args ) );
                  definitionsChanged = true;
                } else {
                  size_t startValue = endName;
                  while( ( startValue < restOfLine.length() ) && IsSpaceOrTab( restOfLine[startValue] ) )
                  { startValue++; }
                  std::string value = restOfLine.substr( startValue );
                  preprocessorDefinitions[key] = value;
                  ppDefineHistory.push_back( PPDefinition( lineCurrent, key, value ) );
                  definitionsChanged = true;
                }
              }
            } else if( sc.Match( "undef" ) ) {
              if( options.updatePreprocessor && !preproc.IsInactive() ) {
                std::string restOfLine = GetRestOfLine( styler, sc.currentPos + 5, true );
                std::vector<std::string> tokens = Tokenize( restOfLine );
                std::string key;
                if( tokens.size() >= 1 ) {
                  key = tokens[0];
                  preprocessorDefinitions.erase( key );
                  ppDefineHistory.push_back( PPDefinition( lineCurrent, key, "", true ) );
                  definitionsChanged = true;
                }
              }
            }
          }
        }
      } else if( isoperator( sc.ch ) ) {
        sc.SetState( SCE_C_OPERATOR | activitySet );
      }
    }
    if( !IsASpace( sc.ch ) && !IsSpaceEquiv( MaskActive( sc.state ) ) ) {
      chPrevNonWhite = sc.ch;
      visibleChars++;
    }
    continuationLine = false;
    sc.Forward();
  }
  const bool rawStringsChanged = rawStringTerminators.Merge( rawSTNew, lineCurrent );
  if( definitionsChanged || rawStringsChanged ) {
    styler.ChangeLexerState( startPos, startPos + length );
  }
  sc.Complete();
}

void  LexerCPP::Fold( unsigned int startPos, int length, int initStyle, IDocument *pAccess ) {
  if( !options.fold ) {
    return;
  }
  LexAccessor styler( pAccess );
  unsigned int endPos = startPos + length;
  int visibleChars = 0;
  bool inLineComment = false;
  int lineCurrent = styler.GetLine( startPos );
  int levelCurrent = SC_FOLDLEVELBASE;
  if( lineCurrent > 0 ) {
    levelCurrent = styler.LevelAt( lineCurrent - 1 ) >> 16;
  }
  unsigned int lineStartNext = styler.LineStart( lineCurrent + 1 );
  int levelMinCurrent = levelCurrent;
  int levelNext = levelCurrent;
  char chNext = styler[startPos];
  int styleNext = MaskActive( styler.StyleAt( startPos ) );
  int style = MaskActive( initStyle );
  const bool userDefinedFoldMarkers = !options.foldExplicitStart.empty() && !options.foldExplicitEnd.empty();
  for( unsigned int i = startPos; i < endPos; i++ ) {
    char ch = chNext;
    chNext = styler.SafeGetCharAt( i + 1 );
    int stylePrev = style;
    style = styleNext;
    styleNext = MaskActive( styler.StyleAt( i + 1 ) );
    bool atEOL = i == ( lineStartNext - 1 );
    if( ( style == SCE_C_COMMENTLINE ) || ( style == SCE_C_COMMENTLINEDOC ) ) {
      inLineComment = true;
    }
    if( options.foldComment && options.foldCommentMultiline && IsStreamCommentStyle( style ) && !inLineComment ) {
      if( !IsStreamCommentStyle( stylePrev ) ) {
        levelNext++;
      } else if( !IsStreamCommentStyle( styleNext ) && !atEOL ) {
        levelNext--;
      }
    }
    if( options.foldComment && options.foldCommentExplicit && ( ( style == SCE_C_COMMENTLINE ) || ( style == SCE_C_WXSMITH ) || options.foldExplicitAnywhere ) ) {
      if( userDefinedFoldMarkers ) {
        if( styler.Match( i, options.foldExplicitStart.c_str() ) ) {
          levelNext++;
        } else if( styler.Match( i, options.foldExplicitEnd.c_str() ) ) {
          levelNext--;
        }
      } else {
        if( ( ch == '/' ) && ( chNext == '/' ) ) {
          char chNext2 = styler.SafeGetCharAt( i + 2 );
          char chNext3 = styler.SafeGetCharAt( i + 3 );
          if( chNext2 == '{' ) {
            levelNext++;
          } else if( chNext2 == '}' ) {
            levelNext--;
          } else if( chNext2 == '(' && chNext3 == '*' ) {
            levelNext++;
          } else if( chNext2 == '*' && chNext3 == ')' ) {
            levelNext--;
          }
        }
      }
    }
    if( options.foldPreprocessor && ( style == SCE_C_PREPROCESSOR ) ) {
      if( ch == '#' ) {
        unsigned int j = i + 1;
        while( ( j < endPos ) && IsASpaceOrTab( styler.SafeGetCharAt( j ) ) ) {
          j++;
        }
        if( styler.Match( j, "region" ) || styler.Match( j, "if" ) ) {
          levelNext++;
        } else if( styler.Match( j, "end" ) ) {
          levelNext--;
        }
      }
    }
    if( options.foldSyntaxBased && ( style == SCE_C_OPERATOR ) ) {
      if( ch == '{' || ch == '[' ) {
        if( levelMinCurrent > levelNext ) {
          levelMinCurrent = levelNext;
        }
        levelNext++;
      } else if( ch == '}' || ch == ']' ) {
        levelNext--;
      }
    }
    if( !IsASpace( ch ) ) {
      visibleChars++;
    }
    if( atEOL || ( i == endPos - 1 ) ) {
      int levelUse = levelCurrent;
      if( options.foldSyntaxBased && options.foldAtElse ) {
        levelUse = levelMinCurrent;
      }
      int lev = levelUse | levelNext << 16;
      if( visibleChars == 0 && options.foldCompact ) {
        lev |= SC_FOLDLEVELWHITEFLAG;
      }
      if( levelUse < levelNext ) {
        lev |= SC_FOLDLEVELHEADERFLAG;
      }
      if( lev != styler.LevelAt( lineCurrent ) ) {
        styler.SetLevel( lineCurrent, lev );
      }
      lineCurrent++;
      lineStartNext = styler.LineStart( lineCurrent + 1 );
      levelCurrent = levelNext;
      levelMinCurrent = levelCurrent;
      if( atEOL && ( i == static_cast<unsigned int>( styler.Length() - 1 ) ) ) {
        styler.SetLevel( lineCurrent, ( levelCurrent | levelCurrent << 16 ) | SC_FOLDLEVELWHITEFLAG );
      }
      visibleChars = 0;
      inLineComment = false;
    }
  }
}

void LexerCPP::EvaluateTokens( std::vector<std::string> &tokens, const SymbolTable &preprocessorDefinitions ) {
  tokens.erase( std::remove_if( tokens.begin(), tokens.end(), OnlySpaceOrTab ), tokens.end() );
  for( size_t i = 0; ( i + 1 ) < tokens.size(); ) {
    if( tokens[i] == "defined" ) {
      const char *val = "0";
      if( tokens[i + 1] == "(" ) {
        if( ( ( i + 2 ) < tokens.size() ) && ( tokens[i + 2] == ")" ) ) {
          tokens.erase( tokens.begin() + i + 1, tokens.begin() + i + 3 );
        } else if( ( ( i + 3 ) < tokens.size() ) && ( tokens[i + 3] == ")" ) ) {
          SymbolTable::const_iterator it = preprocessorDefinitions.find( tokens[i + 2] );
          if( it != preprocessorDefinitions.end() ) {
            val = "1";
          }
          tokens.erase( tokens.begin() + i + 1, tokens.begin() + i + 4 );
        } else
        { tokens.erase( tokens.begin() + i + 1, tokens.begin() + i + 2 ); }
      } else {
        SymbolTable::const_iterator it = preprocessorDefinitions.find( tokens[i + 1] );
        if( it != preprocessorDefinitions.end() ) {
          val = "1";
        }
      }
      tokens[i] = val;
    } else
    { i++; }
  }
  const size_t maxIterations = 100;
  size_t iterations = 0;
  for( size_t i = 0; ( i < tokens.size() ) && ( iterations < maxIterations ); ) {
    iterations++;
    if( setWordStart.Contains( static_cast<unsigned char>( tokens[i][0] ) ) ) {
      SymbolTable::const_iterator it = preprocessorDefinitions.find( tokens[i] );
      if( it != preprocessorDefinitions.end() ) {
        std::vector<std::string> macroTokens = Tokenize( it->second.value );
        if( it->second.IsMacro() ) {
          if( ( i + 1 < tokens.size() ) && ( tokens.at( i + 1 ) == "(" ) ) {
            std::vector<std::string> argumentNames = StringSplit( it->second.arguments, ',' );
            std::map<std::string, std::string> arguments;
            size_t arg = 0;
            size_t tok = i + 2;
            while( ( tok < tokens.size() ) && ( arg < argumentNames.size() ) && ( tokens.at( tok ) != ")" ) ) {
              if( tokens.at( tok ) != "," ) {
                arguments[argumentNames.at( arg )] = tokens.at( tok );
                arg++;
              }
              tok++;
            }
            tokens.erase( tokens.begin() + i, tokens.begin() + tok + 1 );
            macroTokens.erase( std::remove_if( macroTokens.begin(), macroTokens.end(), OnlySpaceOrTab ), macroTokens.end() );
            for( size_t iMacro = 0; iMacro < macroTokens.size(); ) {
              if( setWordStart.Contains( static_cast<unsigned char>( macroTokens[iMacro][0] ) ) ) {
                std::map<std::string, std::string>::const_iterator itFind = arguments.find( macroTokens[iMacro] );
                if( itFind != arguments.end() )
                { macroTokens[iMacro] = itFind->second; }
              }
              iMacro++;
            }
            tokens.insert( tokens.begin() + i, macroTokens.begin(), macroTokens.end() );
          } else
          { i++; }
        } else {
          tokens.erase( tokens.begin() + i );
          tokens.insert( tokens.begin() + i, macroTokens.begin(), macroTokens.end() );
        }
      } else
      { tokens.erase( tokens.begin() + i ); }
    } else
    { i++; }
  }
  BracketPair bracketPair = FindBracketPair( tokens );
  while( bracketPair.itBracket != tokens.end() ) {
    std::vector<std::string> inBracket( bracketPair.itBracket + 1, bracketPair.itEndBracket );
    EvaluateTokens( inBracket, preprocessorDefinitions );
    tokens.insert( bracketPair.itBracket, inBracket.begin(), inBracket.end() );
    bracketPair = FindBracketPair( tokens );
    tokens.erase( bracketPair.itBracket, bracketPair.itEndBracket + 1 );
    bracketPair = FindBracketPair( tokens );
  }
  for( size_t j = 0; ( j + 1 ) < tokens.size(); ) {
    if( setNegationOp.Contains( tokens[j][0] ) ) {
      int isTrue = atoi( tokens[j + 1].c_str() );
      if( tokens[j] == "!" ) {
        isTrue = !isTrue;
      }
      std::vector<std::string>::iterator itInsert =
        tokens.erase( tokens.begin() + j, tokens.begin() + j + 2 );
      tokens.insert( itInsert, isTrue ? "1" : "0" );
    } else
    { j++; }
  }
  enum precedence { precArithmetic, precRelative, precLogical };
  for( int prec = precArithmetic; prec <= precLogical; prec++ ) {
    for( size_t k = 0; ( k + 2 ) < tokens.size(); ) {
      char chOp = tokens[k + 1][0];
      if(
        ( ( prec == precArithmetic ) && setArithmethicOp.Contains( chOp ) ) ||
        ( ( prec == precRelative ) && setRelOp.Contains( chOp ) ) ||
        ( ( prec == precLogical ) && setLogicalOp.Contains( chOp ) )
      ) {
        int valA = atoi( tokens[k].c_str() );
        int valB = atoi( tokens[k + 2].c_str() );
        int result = 0;
        if( tokens[k + 1] == "+" ) {
          result = valA + valB;
        } else if( tokens[k + 1] == "-" ) {
          result = valA - valB;
        } else if( tokens[k + 1] == "*" ) {
          result = valA * valB;
        } else if( tokens[k + 1] == "/" ) {
          result = valA / ( valB ? valB : 1 );
        } else if( tokens[k + 1] == "%" ) {
          result = valA % ( valB ? valB : 1 );
        } else if( tokens[k + 1] == "<" ) {
          result = valA < valB;
        } else if( tokens[k + 1] == "<=" ) {
          result = valA <= valB;
        } else if( tokens[k + 1] == ">" ) {
          result = valA > valB;
        } else if( tokens[k + 1] == ">=" ) {
          result = valA >= valB;
        } else if( tokens[k + 1] == "==" ) {
          result = valA == valB;
        } else if( tokens[k + 1] == "!=" ) {
          result = valA != valB;
        } else if( tokens[k + 1] == "||" ) {
          result = valA || valB;
        } else if( tokens[k + 1] == "&&" ) {
          result = valA && valB;
        }
        char sResult[30];
        sprintf( sResult, "%d", result );
        std::vector<std::string>::iterator itInsert =
          tokens.erase( tokens.begin() + k, tokens.begin() + k + 3 );
        tokens.insert( itInsert, sResult );
      } else
      { k++; }
    }
  }
}

std::vector<std::string> LexerCPP::Tokenize( const std::string &expr ) const {
  std::vector<std::string> tokens;
  const char *cp = expr.c_str();
  while( *cp ) {
    std::string word;
    if( setWord.Contains( static_cast<unsigned char>( *cp ) ) ) {
      while( setWord.Contains( static_cast<unsigned char>( *cp ) ) ) {
        word += *cp;
        cp++;
      }
    } else if( IsSpaceOrTab( *cp ) ) {
      while( IsSpaceOrTab( *cp ) ) {
        word += *cp;
        cp++;
      }
    } else if( setRelOp.Contains( static_cast<unsigned char>( *cp ) ) ) {
      word += *cp;
      cp++;
      if( setRelOp.Contains( static_cast<unsigned char>( *cp ) ) ) {
        word += *cp;
        cp++;
      }
    } else if( setLogicalOp.Contains( static_cast<unsigned char>( *cp ) ) ) {
      word += *cp;
      cp++;
      if( setLogicalOp.Contains( static_cast<unsigned char>( *cp ) ) ) {
        word += *cp;
        cp++;
      }
    } else {
      word += *cp;
      cp++;
    }
    tokens.push_back( word );
  }
  return tokens;
}

bool LexerCPP::EvaluateExpression( const std::string &expr, const SymbolTable &preprocessorDefinitions ) {
  std::vector<std::string> tokens = Tokenize( expr );
  EvaluateTokens( tokens, preprocessorDefinitions );
  bool isFalse = tokens.empty() ||
                 ( ( tokens.size() == 1 ) && ( ( tokens[0] == "" ) || tokens[0] == "0" ) );
  return !isFalse;
}

LexerModule lmCPP( SCLEX_CPP, LexerCPP::LexerFactoryCPP, "cpp", cppWordLists );
LexerModule lmCPPNoCase( SCLEX_CPPNOCASE, LexerCPP::LexerFactoryCPPInsensitive, "cppnocase", cppWordLists );
